# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy
import datetime
import re
from scrapy.loader.processors import MapCompose, TakeFirst, Join
from scrapy.loader import ItemLoader

'''
    将爬取的数据从非结构化转换为结构化
    yield的时候，scrapy会识别items的实例，将item直接路由到pipelines。然后在pipelines里做去重和保存相关的内容
'''


class KkspiderItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass


def add_title_from(value):
    return value + "_jobbole"


def date_convert(value):
    try:
        create_time = datetime.datetime.strptime(value, "%Y/%m/%d").date()
    except Exception as e:
        create_time = datetime.datetime.now().date()
    return create_time


def regex_match_nums(value):
    re_match_collect = re.match(".*(\d+).*", value)
    if re_match_collect:
        nums = int(re_match_collect.group(1))
    else:
        nums = 0
    return nums


def remove_tags_dont_need(value):
    # 去掉tag中提取的不需要的评论字段
    if "评论" in value:
        return ""
    else:
        return value


def return_value_not_takefirst(value):
    return value


# 重载itemLoader，实现自己的loader方式
class ArticleItemLoader(ItemLoader):
    # 自定义ItemLoader,这样就不需要每一次都要手动output_processor为TakeFirst了
    # 使用TakeFirst用来获取传入list的第一个，相当于原来的extract_first()
    default_output_processor = TakeFirst()


class ArticleItem(scrapy.Item):
    # field表示任何情况都可以接收
    title = scrapy.Field(
        input_processor=MapCompose(lambda x: x + "-jobbole")
    )
    create_time = scrapy.Field(
        input_processor=MapCompose(date_convert)
    )
    url = scrapy.Field()
    url_object_id = scrapy.Field()
    from_image_url = scrapy.Field(
        output_processor=MapCompose(return_value_not_takefirst)
    )
    from_image_path = scrapy.Field()
    praise_num = scrapy.Field(
        input_processor=MapCompose(regex_match_nums)
    )
    collect_num = scrapy.Field(
        input_processor=MapCompose(regex_match_nums)
    )
    comment_num = scrapy.Field(
        input_processor=MapCompose(regex_match_nums)
    )
    tags = scrapy.Field(
        input_processor=MapCompose(remove_tags_dont_need),
        output_processor=Join(",")
    )

    # 将sql语句放在item，使得在pipeline中可以统一填充使得代码复用
    def get_sql(self):
        insert_sql = """
                       insert into article(title,create_time,url,url_object_id,from_image_url,from_image_path,praise_num,collect_num,comment_num,tags) 
                       VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) 
                   """
        params = (
            self["title"], self["create_time"], self["url"], self["url_object_id"], self["from_image_url"],
            self["from_image_path"], self["praise_num"], self["collect_num"], self["comment_num"], self["tags"])
        return insert_sql, params

    pass


class juejinJsonItem(scrapy.Item):
    collectionCount = scrapy.Field()
    commentsCount = scrapy.Field()
    updatedAt = scrapy.Field()
    originalUrl = scrapy.Field()
    tags = scrapy.Field()
    title = scrapy.Field()
    viewsCount = scrapy.Field()
    summaryInfo = scrapy.Field()
    screenshot = scrapy.Field()

    def get_sql(self):
        insert_sql = """
                       insert into juejin(title,summaryInfo,updatedAt,tags,collectionCount,commentsCount,viewsCount,originalUrl,screenshot) 
                       VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) 
                   """
        params = (
            self["title"], self["summaryInfo"], self["updatedAt"], self["tags"], self["collectionCount"],
            self["commentsCount"], self["viewsCount"], self["originalUrl"], self["screenshot"])
        return insert_sql, params

    pass


class lagouItem(scrapy.Item):
    title = scrapy.Field()
    url = scrapy.Field()
    url_object_id = scrapy.Field()
    salary = scrapy.Field()
    job_city = scrapy.Field()
    work_years = scrapy.Field()
    degree_need = scrapy.Field()
    job_type = scrapy.Field()
    job_advance = scrapy.Field()
    job_addr = scrapy.Field()
    job_desc = scrapy.Field()
    publish_time = scrapy.Field()
    company_name = scrapy.Field()
    company_url = scrapy.Field()
    tags = scrapy.Field()
    crawl_time = scrapy.Field()
